import scrapy
# import json
from myScrapy.items import DoubanCommentsItem
# from scrapy.selector import Selector
from scrapy.http import Request
# from myScrapy.settings import USER_AGENTS

# from fake_useragent import UserAgent

class DoubanSpider(scrapy.Spider):
    name = "douban"
    allowed_domains = ["douban.com"]
    api = "https://movie.douban.com/subject/1889243/comments"
    start_urls = [
        "https://movie.douban.com/subject/1889243/comments?start=0&limit=20&sort=new_score&status=P&percent_type=",
    ]

    custom_settings = dict(
        DOWNLOAD_DELAY=5,
        CONCURRENT_REQUESTS_PER_DOMAIN=8,
        CONCURRENT_REQUESTS_PER_IP=8,
        ROBOTSTXT_OBEY = False,
        DOWNLOADER_MIDDLEWARES={
            'myScrapy.middlewares.MyUseragentMiddleware': 1000,
        },
        ITEM_PIPELINES = {
            'myScrapy.pipelines.DoubanCommentsPipeline': 300,
        }
    )

    def parse(self, response):
        # json_obj = json.loads(response.text)
        # html_str = json_obj['html']
        # json_response = Selector(text=html_str)
        for sel in response.xpath('//div[@class="comment-item"]'):
            comment = sel.xpath('div[@class="comment"]//span[@class="short"]/text()').extract()
            comment_id = sel.xpath('@data-cid').extract()
            people_name = sel.xpath('div[@class="comment"]//span[@class="comment-info"]/a/text()').extract()
            people_url = sel.xpath('div[@class="comment"]//span[@class="comment-info"]/a/@href').extract()
            score = sel.xpath('div[@class="comment"]//span[contains(@class, "rating")]/@class').extract()
            comment_time = sel.xpath('div[@class="comment"]//span[contains(@class, "comment-time")]/@title').extract()
            votes = sel.xpath('div[@class="comment"]//span[@class="votes"]/text()').extract()

            item = DoubanCommentsItem()
            item['comment'] = comment[0]
            item['comment_id'] = int(comment_id[0])
            item['object_id'] = 1889243
            item['people_name'] = people_name[0]
            item['people_url'] = people_url[0]
            if score:
                item['score'] = score[0][7:9]
            else:
                item['score'] = 0
            
            item['comment_time'] = comment_time[0]
            item['votes'] = votes[0]
            yield item

        next_link_params = response.xpath('//a[@class="next"]/@href').extract()
        if next_link_params:
            print(next_link_params)
            next_link_url = self.api + next_link_params[0]
            print(next_link_url)
            yield scrapy.Request(next_link_url, callback=self.parse) 